#!/usr/bin/env python
# -*- coding: utf-8 -*-
# author：albert time:2019/7/26
import  pandas as pd
import  requests
import  re
import  math

'''获取所有列表页面'''
user_list_url = "http://www.moko.cc/post/{}/list.html"
#获取所有用户的列表页
user_profiles = []
all_pages = []
# 美空网网址
base_url = "http://www.moko.cc{}"
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
def read_data():
    df = pd.read_csv('moko70000.csv')
    #去掉重复的呢称数据
    df = df.drop_duplicates(['nikename'])
    #按照粉丝数目进行降序
    profiles = df.sort_values('follows',ascending=False)['profile']
    for i in profiles:
        #拼接连接
        user_profiles.append(user_list_url.format(i))

'''获取图片的列表页面'''
def get_img_list_page(user_url):

    # test_url = "http://www.moko.cc/post/da39db43246047c79dcaef44c201492d/list.html"
    try:
        response = requests.get(user_url, headers=headers, timeout=3)
    except Exception as e:
        print(e)
        return
    page_text = response.text
    pattern = re.compile('<p class="title"><a hidefocus="ture".*?href="(.*?)" class="mwC u">.*?\((\d+?)\)</a></p>')
    # 获取page_list
    page_list = pattern.findall(page_text)
    for page in page_list:
        if page[1] == '0':
            page_list.remove(page)
            continue
        else:
            get_all_list_page(page[0],page[1])


def get_all_list_page(start_page,totle):
    #math.ceil 函数返回数字的上入整数(加一取整)
    page_count = math.ceil(int(totle)/28)+1
    for i in range(1,page_count):
        pages = re.sub(r'\d+?\.html',str(i)+'.html',start_page)
        all_pages.append(base_url.format(pages))
    print('已经抓取到{}条'.format(len(all_pages)))
    if (len(all_pages)>1000):
        pd.DataFrame(all_pages).to_csv("pages.csv",mode='a+')
        all_pages.clear()

if __name__ == '__main__':
    read_data()
    # print(user_profiles)
    for user in user_profiles:
        get_img_list_page(user)